package com.spider.service;

import java.util.List;

import javax.management.JMException;

import com.spider.dao.HBaseDAOImp;
import com.spider.dao.HBaseDao;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.monitor.SpiderMonitor;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler;

public class QuesPageProcessPhy implements PageProcessor{

	private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);
	private HBaseDao dao = new HBaseDAOImp();
	private int index = 0;
	
	@Override
	public Site getSite() {
		// TODO Auto-generated method stub
		return site;
	}
	
	@Override
	public void process(Page page) {
		// TODO Auto-generated method stub
		page.putField("html", page.getHtml());
		page.putField("url", page.getUrl());
		
		String html = page.getResultItems().get("html").toString();
		String urled = page.getResultItems().get("url").toString();
			
		List<String> urls = page.getHtml()
				.xpath("//div[@class='laypage_main laypageskin_default']")
				.links().regex(".*hdSearch=.*").all();
		
		for(int i = 0;i < urls.size();i++) {
			dao.insert("urlPhy", "url"+index+"--"+urls.get(i), "url", "url", urls.get(i));
			index++;
		}
			
		List<String> urlz = page.getHtml()
				.xpath("//dl[@class='z-open']")
				.links().regex(".*kid=.*").all();	
		
		for(int i = 0;i < urlz.size();i++) {
			dao.insert("urlPhy", "url"+index+"--"+urlz.get(i), "url", "url", urlz.get(i));
			index++;
		}
		
		List<String> urlBook = page.getHtml()
				.xpath("//div[@id=filter-book]")
				.links().regex(".*bid=.*").all();
		for (int i = 0; i < urlBook.size(); i++) {
			dao.insert("urlPhy", "url"+index+"--"+urlBook.get(i), "url", "url", urlBook.get(i));
			index++;
		}
		
		List<String> urlType = page.getHtml()
				.xpath("//div[@id=fitler-tixing]")
				.links().regex(".*qtid=.*").all();
		
		for (int i = 0; i < urlType.size(); i++) {
			dao.insert("urlPhy", "url"+index+"--"+urlType.get(i), "url", "url", urlType.get(i));
			index++;
		}
		
		List<String> urlVersion = page.getHtml()
				.xpath("//div[@id=filter-version]")
				.links().regex(".*vid=.*").all();
		for (int i = 0; i < urlVersion.size(); i++) {
			dao.insert("urlPhy", "url"+index+"--"+urlVersion.get(i), "url", "url", urlVersion.get(i));
			index++;
		}
		
		dao.insert("htmlPhy", "content"+(index++)+"", "base", "content", html);
		dao.insert("urlPhy", "urled"+index+"--"+urled, "urled", "urled", urled);

		System.err.println("urlPhy:"+index);
		
		page.addTargetRequests(urls);
		page.addTargetRequests(urlz);
		page.addTargetRequests(urlBook);
		page.addTargetRequests(urlType);
		page.addTargetRequests(urlVersion);
	}
	

	public static void main(String[] args) {
		//overing
		Spider spider = Spider.create(new QuesPageProcessPhy())
		.setScheduler(new FileCacheQueueScheduler("./phyUrlManager"))
		.addUrl("http://www.tiku.com/testPaper.html?sct=0&cn=%E7%89%A9%E7%90%86&st=2&cid=500011&bid=800074&vid=800013&uid=800335&cptid=701041&kid=700344&qtid=600062&difficulty=&sort=0")
		.thread(5);
		
		try {
			SpiderMonitor.instance().register(spider);
		} catch (JMException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		spider.run();
	}
}
